# coding:utf-8
from abc import ABCMeta, abstractmethod
from agent import RandAgent
import requests
"""
一个用于给自定义爬虫继承的抽象爬虫类Spider
"""


class Spider(metaclass=ABCMeta):
    # 类属性db
    db = None

    @classmethod
    def get(cls, url, is_text=True, is_encoding=False, headers={}):
        if not headers:
            headers = {
                'User-Agent': RandAgent.random()
            }
        response = requests.get(url, headers=headers)
        if is_encoding:
            response.encoding = response.apparent_encoding
        return response.text if is_text else response.content

    @abstractmethod
    def run(self):
        pass

    @abstractmethod
    def parse_all_page(self, url):
        pass

    @abstractmethod
    def parse_one_page(self, url):
        pass

    @abstractmethod
    def parse_nav(self, url):
        """根据url解析导航条"""
        pass

    @abstractmethod
    def save_to_sqlite(self):
        pass


